# -*- coding: utf-8 -*-
# @Time    : 2024/1/28 21:35
# @Author  : micah
# @File    : 11.使用多线程完成爱奇艺爬虫.py
# @Software: PyCharm


import pymongo
import requests
import threading
from queue import Queue


class AiQiYi:
    def __init__(self):
        self.mongo_client = pymongo.MongoClient()
        self.collection = self.mongo_client['py_spider']['thread_aiqiyi']

        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36',
            'Referer': 'https://list.iqiyi.com/www/2/15-------------11-1-1-iqiyi--.html?s_source=PCW_SC'
        }
        self.api_url = 'https://pcw-api.iqiyi.com/search/recommend/list?channel_id=2&data_type=1&mode=11&page_id={}&ret_num=48&session=85dd981b69cead4b60f6d980438a5664&three_category_id=15;must'

        # 创建队列
        self.url_queue = Queue()
        self.json_queue = Queue()
        self.content_dict_queue = Queue()

    # 获取url
    def get_url(self):
        for page in range(1, 6):
            self.url_queue.put(self.api_url.format(page))

    # 发送请求并获取json数据
    def get_api_json(self):
        while True:
            url = self.url_queue.get()
            response = requests.get(url, headers=self.headers).json()
            self.json_queue.put(response)
            self.url_queue.task_done()  # 让队列中的计数器减1

    # 数据解析
    def parse_movie_info(self):
        while True:
            json_data = self.json_queue.get()
            category_movies = json_data['data']['list']
            for movie in category_movies:
                item = dict()
                item['title'] = movie['title']
                item['playUrl'] = movie['playUrl']
                item['description'] = movie['description']

                # 将组织好的字典数据上传到content_dict_queue
                self.content_dict_queue.put(item)
            self.json_queue.task_done()

    # 数据保存
    def save_movie_info(self):
        while True:
            item = self.content_dict_queue.get()
            self.collection.insert_one(item)
            print('插入成功:', item)
            self.content_dict_queue.task_done()

    # 启动函数
    def main(self):
        # 创建线程对象列表
        thread_list = list()

        t_url = threading.Thread(target=self.get_url)
        thread_list.append(t_url)

        for _ in range(3):
            t_get_json = threading.Thread(target=self.get_api_json)
            thread_list.append(t_get_json)

        t_parse_info = threading.Thread(target=self.parse_movie_info)
        thread_list.append(t_parse_info)

        # 对于保存方法建议无论在什么项目中都使用一个线程去完成
        t_save_info = threading.Thread(target=self.save_movie_info)
        thread_list.append(t_save_info)

        # 循环线程列表并启动线程
        for thread_obj in thread_list:
            thread_obj.daemon = True
            thread_obj.start()

        # 判断所有队列中的计数器是否为0, 如果都为0, 则释放主线程
        for queue in [self.url_queue, self.json_queue, self.content_dict_queue]:
            queue.join()

        self.mongo_client.close()
        print('数据库链接对象已关闭...')
        print('主线程结束...')


if __name__ == '__main__':
    aqy = AiQiYi()
    aqy.main()
